# -*- coding: utf-8 -*-
# @Time    : 2019/10/8 17:19
# @Author  : ZSQ
# @Email   : zsq199170918@163.com
# @FileName: zhongshan.py
# @Software: PyCharm
import scrapy

import re
import logging
from copy import copy, deepcopy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem

logger = logging.getLogger(__name__)


class ZhongshanSpider(scrapy.Spider):
    name = 'zhongshan'
    allowed_domains = ['113.106.13.237']
    start_urls = ['http://113.106.13.237:82/']
    room_li_temp_url_p = 'http://113.106.13.237:82/Common/Agents/ExeFunCommon.aspx?{}&req={}'  # POST  房号列表url
    room_li_temp_url_g = 'http://113.106.13.237:82/HPMS/RoomList.aspx?code={}&rsr=1000&rse=0&rhx=3000&jzmj=&tnmj='  # GET  房号列表url
    page_num_set = set()  # 用于做翻页处理，保存页数
    page_resp_dict = dict()  # 用于做翻页处理，保存翻页响应

    def parse(self, response):
        # 获取当前页参数
        current_page = response.xpath("//input[@name='PageNavigator1$txtNewPageIndex']/@value").extract_first()
        total_page = response.xpath("//span[@id='PageNavigator1_LblPageCount']/text()").extract_first()
        VIEWSTATE = response.xpath("//input[@id='__VIEWSTATE']/@value").extract_first()
        VIEWSTATEGENERATOR = response.xpath("//input[@id='__VIEWSTATEGENERATOR']/@value").extract_first()
        EVENTVALIDATION = response.xpath("//input[@id='__EVENTVALIDATION']/@value").extract_first()
        # 获取当前页许可证分组
        tr_li = response.xpath("//div[@class='resultlist']/table//tr[position()>1]")
        # 保存当前页响应
        if not self.page_resp_dict.get(current_page):
            logger.info('>>>>>>>>第{}/{}页<<<<<<<<'.format(current_page, total_page))
            self.page_resp_dict[current_page] = tr_li
        # 判断是否翻页完成
        if len(self.page_resp_dict) == int(total_page):
            for page_str, tr_li in self.page_resp_dict.items():
                if len(tr_li):
                    for tr in tr_li:
                        item_eg = FdcEstateGuidItem()
                        item_eg['preSalePermit'] = tr.xpath("./td[2]/a//text()").extract_first()
                        item_eg['preSalePermit'] = item_eg['preSalePermit'] if item_eg['preSalePermit'] and item_eg[
                            'preSalePermit'] != '空' else None
                        permit_url = tr.xpath("./td[2]/a/@href").extract_first()
                        permit_id_li = re.findall(r'\d+', permit_url)
                        item_eg['permitId'] = permit_id_li[0] if len(permit_id_li) else None
                        item_eg['developerName'] = tr.xpath("./td[3]//text()").extract_first()
                        item_eg['projectName'] = tr.xpath("./td[4]//text()").extract_first()
                        item_eg['estateAddress'] = tr.xpath("./td[5]//text()").extract_first()
                        item_eg['districtName'] = tr.xpath("./td[7]//text()").extract_first()
                        item_eg['totalRoomNo'] = tr.xpath("./td[8]//text()").extract_first()
                        item_eg['saleableRoomNo'] = tr.xpath("./td[9]//text()").extract_first()
                        yield response.follow(
                            permit_url,
                            callback=self.parse_project,
                            meta=dict(item_eg=deepcopy(item_eg)),
                            priority=10,
                        )
                else:
                    logger.warning('第{}页许可证列表页获取为空'.format(page_str))

        # 翻页
        for page_num in range(2, int(total_page) + 1):
            self.page_num_set.add(1)  # 添加首页
            len_before = len(self.page_num_set)
            self.page_num_set.add(page_num)
            len_after = len(self.page_num_set)
            if len_after > len_before:
                # 构造请求头
                headers = {
                    'Content-Type': 'application/x-www-form-urlencoded',
                    'Accept-Language': 'zh-CN,zh;q=0.9',
                    'Accept-Encoding': 'gzip, deflate',
                }
                # 构造POST请求参数
                data = {
                    '__EVENTTARGET': 'PageNavigator1$LnkBtnGoto',
                    '__VIEWSTATE': VIEWSTATE,
                    '__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
                    '__EVENTVALIDATION': EVENTVALIDATION,
                    'PageNavigator1$txtNewPageIndex': str(page_num),
                    '__EVENTARGUMENT': '',
                    'txtXMMC': '',
                    'txtXkzh': '',
                    'txtKFQY': '',
                    'txtXMDZ': '',
                    'dpSZQY': '',
                    'txtQSTime': '',
                    'txtZZTime': '',
                }
                yield scrapy.FormRequest(
                    ZhongshanSpider.start_urls[0],
                    headers=headers,
                    formdata=data,
                    priority=100,
                    dont_filter=True,
                )

        """网站机制更新后修改翻页规则，解决500报错    ---By ZSQ 2021/1/18"""
        # if int(current_page) < int(total_page):
        #     # 构造POST请求参数
        #     data = {
        #         '__EVENTTARGET': 'PageNavigator1$LnkBtnGoto',
        #         '__VIEWSTATE': VIEWSTATE,
        #         '__VIEWSTATEGENERATOR': VIEWSTATEGENERATOR,
        #         '__EVENTVALIDATION': EVENTVALIDATION,
        #         'PageNavigator1$txtNewPageIndex': str(int(current_page) + 1),
        #         '__EVENTARGUMENT': '',
        #         'txtXMMC': '',
        #         'txtXkzh': '',
        #         'txtKFQY': '',
        #         'txtXMDZ': '',
        #         'dpSZQY': '',
        #         'txtQSTime': '',
        #         'txtZZTime': '',
        #     }
        #     yield scrapy.FormRequest(
        #         ZhongshanSpider.start_urls[0],
        #         formdata=data,
        #         priority=100,
        #     )

    def parse_project(self, response):
        """
        获取项目详情
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['estateTotalArea'] = response.xpath("//span[@id='Project_GHZJZMJ']/text()").extract_first()
        item_eg['qualificationCertificateNo'] = response.xpath("//span[@id='lblZZZS']/text()").extract_first()
        item_eg['qualificationGrade'] = response.xpath("//span[@id='lblZZDJ']/text()").extract_first()
        item_eg['certDate'] = response.xpath("//span[@id='Presel_FZRQ']/text()").extract_first()
        item_eg['certStartDate'] = response.xpath("//span[@id='Presel_YXQX1']/text()").extract_first()
        item_eg['certExpiryDate'] = response.xpath("//span[@id='Presel_YXQX2']/text()").extract_first()
        item_eg['CA'] = response.xpath("//span[@id='Presel_FZJG']/text()").extract_first()
        item_eg['preSaleDepositBank'] = response.xpath("//span[@id='Presel_YSZJKHYH']/text()").extract_first()
        item_eg['preSaleAccountNum'] = response.xpath("//span[@id='Presel_YSZJZH']/text()").extract_first()
        yield item_eg

        # 获取楼栋分组
        span_li = response.xpath("//span[@for]")
        if len(span_li):
            for span in span_li:
                item_bd = FdcBuildingItem()
                item_bd['districtName'] = item_eg['districtName']
                item_bd['projectName'] = item_eg['projectName']
                item_bd['permitId'] = item_eg['permitId']
                item_bd['preSalePermit'] = item_eg['preSalePermit']
                item_bd['blockName'] = span.xpath("./text()").extract_first()
                item_bd['buildingId'] = span.xpath("./preceding-sibling::input[1]/@bid").extract_first()
                yield item_bd

                """方式一：构造POST请求参数(数据较少，改用方式二)"""
                # body_temp_str = '''<?xml version="1.0" encoding="utf-8" standalone="yes"?>\n<param funname="CBuildTableV2.GetBuildHTMLEx">\n<item>../</item>\n<item>{}</item>\n<item>1</item>\n<item>1</item>\n<item>80</item>\n<item>840</item>\n<item>g_oBuildTable</item>\n<item> 1=1</item>\n<item>1</item>\n<item></item>\n<item></item>\n</param>\n'''
                # body_str = body_temp_str.format(item_bd['buildingId'])
                # decimal_num = random.random()
                # time_stamp = int(round(datetime.datetime.now().timestamp()))
                # room_li_url = ZhongshanSpider.room_li_temp_url_p.format(decimal_num, time_stamp)
                # yield scrapy.Request(
                #     room_li_url,
                #     method='POST',
                #     body=body_str,
                #     callback=self.parse_room_li,
                #     meta=dict(item_bd=deepcopy(item_bd)),
                # )
                """方式二：GET请求"""
                room_li_url = ZhongshanSpider.room_li_temp_url_g.format(item_bd['buildingId'])
                yield scrapy.Request(
                    room_li_url,
                    callback=self.parse_room_li,
                    meta=dict(item_bd=deepcopy(item_bd)),
                    priority=20,
                )
        else:
            logger.warning(
                '{}  {}-{}-{}-{} 获取楼栋列表为空'.format(response.request.url, item_eg['districtName'], item_eg['projectName'],
                                                  item_eg['permitId'], item_eg['preSalePermit']))

    def parse_room_li(self, response):
        """
        获取房号列表
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        # 获取房号分组
        tr_li = response.xpath("//table//tr[position()>1]")
        if len(tr_li):
            for tr in tr_li:
                item_rm = FdcRoomItem()
                item_rm['districtName'] = item_bd['districtName']
                item_rm['projectName'] = item_bd['projectName']
                item_rm['blockName'] = item_bd['blockName']
                item_rm['buildingId'] = item_bd['buildingId']
                item_rm['roomNo'] = tr.xpath("./td[1]//text()").extract_first()
                item_rm['unitName'] = tr.xpath("./td[2]//text()").extract_first()
                item_rm['roomUse'] = tr.xpath("./td[3]//text()").extract_first()
                item_rm['roomArea'] = tr.xpath("./td[4]//text()").extract_first()
                item_rm['innerArea'] = tr.xpath("./td[5]//text()").extract_first()
                item_rm['saleStatus'] = tr.xpath("./td[6]//text()").extract_first()
                room_info_url = tr.xpath("./td[1]/a/@href").extract_first()
                if room_info_url:
                    yield response.follow(
                        room_info_url,
                        callback=self.parse_room,
                        meta=dict(item_rm=deepcopy(item_rm)),
                        priority=30,
                    )
                else:
                    item_rm['roomFloor'] = None
                    item_rm['roomStructure'] = '其它'
                    item_rm['roomLocation'] = None
                    yield item_rm
        else:
            logger.warning(
                '{}  {}-{}-{} 获取房号列表为空'.format(response.request.url, item_bd['districtName'], item_bd['projectName'],
                                               item_bd['blockName']))

    def parse_room(self, response):
        """
        获取房号详情
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        item_rm['roomFloor'] = response.xpath("//span[@id='ROOM_MYC']/text()").extract_first()
        item_rm['roomStructure'] = response.xpath("//span[@id='ROOM_HX']/text()").extract_first()
        item_rm['roomLocation'] = response.xpath("//span[@id='ROOM_FWZL']/text()").extract_first()
        yield item_rm
